package org.ntu.rtsearch.analysis;

import java.util.ArrayList;
import java.util.List;

import org.tartarus.snowball.SnowballStemmer;

/**
 * @author fuhao(email: hfhimage@gmail.com)
 * @date 2011-7-24
 */
public class TokenStream {
		
	private static SnowballStemmer stemmer;

	private static StopwordFilter stopwordFilter = new StopwordFilter();
	
	private List<String> tokens = new ArrayList<String>();
	
	private int tokenIdx = 0; 

	public TokenStream(String str) throws Exception {
		initStemmer();
		str = filterLink(str);
		
		char ch;
		StringBuilder sb = new StringBuilder();
		String word = null;
		boolean allUpper = true;
		for(int i = 0; i < str.length(); i++) {
			ch = str.charAt(i);
			
			if(Character.isLetter(ch)) {
				sb.append(ch);
				if(Character.isLowerCase(ch))
					allUpper = false;
			} else if(sb.length() > 0){
				word = sb.toString();
				if(!allUpper) 
					word = word.toLowerCase();
				
				if(!stopwordFilter.isStopword(word)) {
					// to improve, not thread safe
					stemmer.setCurrent(word);	
					stemmer.stem();
					tokens.add(stemmer.getCurrent());
				}
				sb.delete(0, sb.length());
				allUpper = true;
			}
		}
		
		if(sb.length() > 0){
			word = sb.toString();
			if(!allUpper) 
				word = word.toLowerCase();
			
			if(!stopwordFilter.isStopword(word)) {
				// to improve, not thread safe
				stemmer.setCurrent(word);	
				stemmer.stem();
				tokens.add(stemmer.getCurrent());
			}
			sb.delete(0, sb.length());
		}
	}


	private String filterLink(String str) {
		int start, idx;
		while((start = str.indexOf("http://")) != -1) {
			idx = start + 7;
			while(idx < str.length() && !Character.isWhitespace(str.charAt(idx++)))
				;
			str = str.substring(0, start) + str.substring(idx);
		}
		
		// System.out.println(str);
		return str;
	}
	
		
	public String next() {
		if(tokenIdx >= tokens.size())
			return null;
		return tokens.get(tokenIdx++);
	}
	
	public synchronized void initStemmer() throws Exception {
		if(stemmer != null) {
			return;
		}
		@SuppressWarnings("rawtypes")
		Class stemClass = Class.forName("org.tartarus.snowball.ext.englishStemmer");
		stemmer = (SnowballStemmer) stemClass.newInstance();
	}
	
	public static void main(String[] args) throws Exception {
		TokenStream stream = new TokenStream("@fuhao, or #You's# RT http://fdsafsda.fsd.com/fsd.com/fds.jsp beautifully so, FUHAO.$#@!fuhao,SYSU http://fdsafsda.fsd.com/fsd.com/fds.jsp");
		String token = null;
		while((token = stream.next()) != null) {
			System.out.println(token);
		}
	}
}
